{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "接course"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### R4.2.1 设计DL模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入相关包\n",
    "import numpy as np\n",
    "import scipy.signal\n",
    "from gym.spaces import Box, Discrete\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.distributions.normal import Normal\n",
    "from torch.distributions.categorical import Categorical\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义通用cnn model作为base model\n",
    "class cnn_model(nn.Module):\n",
    "    def __init__(self, num_inputs, num_out, activation=nn.ReLU): \n",
    "        super(cnn_model, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(num_inputs, 32, 3, stride=2, padding=1) #卷积层\n",
    "        self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)  #卷积层\n",
    "        self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)  #卷积层\n",
    "        self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1)  #卷积层\n",
    "        self.liner = nn.Linear(32 * 6 * 6, 512) #线性层\n",
    "        self.fc_out = nn.Linear(512, num_out)   #输出层\n",
    "        self._initialize_weights() #模型权重初始化\n",
    "\n",
    "    #对模型参数进行初始化，合理的初始化对训练的收敛起到非常好的作用\n",
    "    def _initialize_weights(self):\n",
    "        for module in self.modules():\n",
    "            if isinstance(module, nn.Conv2d) or isinstance(module, nn.Linear):\n",
    "                nn.init.xavier_uniform_(module.weight)\n",
    "                nn.init.constant_(module.bias, 0)\n",
    "            elif isinstance(module, nn.LSTMCell):\n",
    "                nn.init.constant_(module.bias_ih, 0)\n",
    "                nn.init.constant_(module.bias_hh, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.conv4(x))\n",
    "        x = x.view(x.size(0), -1) #把卷积输出的高纬数据拉平\n",
    "        x = F.relu(self.liner(x))\n",
    "        out = self.fc_out(x)\n",
    "        return out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### R4.2.2 设计ppo的ac模型（Actor/Critic）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Actor负责policy，输出具体要执行的action， Critic负责预测该状态下的value 指引action更新policy \n",
    "训练的时候我们通过Critic的指引来更新Actor，而Critic是由Reward指引更新的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义actor，负责输出action的概率分布，对应简化版的pi函数\n",
    "class userActor(nn.Module):\n",
    "\n",
    "    def __init__(self, obs_dim, act_dim, activation):\n",
    "        super().__init__()\n",
    "        self.logits_net = cnn_model(obs_dim, act_dim, activation=activation) #定义策略模型，输出为动作空间大小\n",
    "        print(self.logits_net)\n",
    "\n",
    "    #计算策略分布，和action的概率\n",
    "    def forward(self, obs, act=None):\n",
    "        pi = Categorical(logits=self.logits_net(obs))\n",
    "        logp_a = None\n",
    "        if act is not None:\n",
    "            logp_a = pi.log_prob(act)\n",
    "        return pi, logp_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义Critic，对应简化板的v函数，训练的时候我们通过Critic的指引来更新Actor，而Critic是由Reward指引更新的\n",
    "class userCritic(nn.Module):\n",
    "\n",
    "    def __init__(self, obs_dim, activation):\n",
    "        super().__init__()\n",
    "        self.v_net = cnn_model(obs_dim, 1, activation=activation) #输出为1，因为输出为当前策略的value预测值（一维）\n",
    "        print(self.v_net)\n",
    "\n",
    "    def forward(self, obs):\n",
    "        return torch.squeeze(self.v_net(obs), -1) # Critical to ensure v has right shape.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### R4.2.3 定义ppo 算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.optim import Adam\n",
    "import gym\n",
    "import time\n",
    "import scipy.signal\n",
    "#from core import userCritic, userActor\n",
    "from env import create_train_env\n",
    "from spinup.utils.logx import EpochLogger\n",
    "from spinup.utils.mpi_pytorch import setup_pytorch_for_mpi, sync_params, mpi_avg_grads\n",
    "from spinup.utils.mpi_tools import mpi_fork, mpi_avg, proc_id, mpi_statistics_scalar, num_procs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#指定使用gpu\n",
    "device = torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#utils:\n",
    "def count_vars(module):\n",
    "    return sum([np.prod(p.shape) for p in module.parameters()])\n",
    "\n",
    "def discount_cumsum(x, discount):\n",
    "    return scipy.signal.lfilter([1], [1, float(-discount)], x[::-1], axis=0)[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def combined_shape(length, shape=None):\n",
    "    if shape is None:\n",
    "        return (length,)\n",
    "    return (length, shape) if np.isscalar(shape) else (length, *shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### R4.2.3.1 定义PPOBuffer 用来存储交互数据，提供给模型训练使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PPOBuffer:\n",
    "    \"\"\"\n",
    "    A buffer for storing trajectories experienced by a PPO agent interacting\n",
    "    with the environment, and using Generalized Advantage Estimation (GAE-Lambda)\n",
    "    for calculating the advantages of state-action pairs.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):\n",
    "        self.obs_buf = np.zeros(combined_shape(size, obs_dim), dtype=np.float32)\n",
    "        self.act_buf = np.zeros(combined_shape(size, act_dim), dtype=np.float32)\n",
    "        self.adv_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.rew_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.ret_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.val_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.logp_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.gamma, self.lam = gamma, lam\n",
    "        self.ptr, self.path_start_idx, self.max_size = 0, 0, size\n",
    "\n",
    "    def store(self, obs, act, rew, val, logp):\n",
    "        \"\"\"\n",
    "        Append one timestep of agent-environment interaction to the buffer.\n",
    "        \"\"\"\n",
    "        assert self.ptr < self.max_size     # buffer has to have room so you can store\n",
    "        self.obs_buf[self.ptr] = obs\n",
    "        self.act_buf[self.ptr] = act\n",
    "        self.rew_buf[self.ptr] = rew\n",
    "        self.val_buf[self.ptr] = val\n",
    "        self.logp_buf[self.ptr] = logp\n",
    "        self.ptr += 1\n",
    "\n",
    "    def finish_path(self, last_val=0):\n",
    "        \"\"\"\n",
    "        Call this at the end of a trajectory, or when one gets cut off\n",
    "        by an epoch ending. This looks back in the buffer to where the\n",
    "        trajectory started, and uses rewards and value estimates from\n",
    "        the whole trajectory to compute advantage estimates with GAE-Lambda,\n",
    "        as well as compute the rewards-to-go for each state, to use as\n",
    "        the targets for the value function.\n",
    "\n",
    "        The \"last_val\" argument should be 0 if the trajectory ended\n",
    "        because the agent reached a terminal state (died), and otherwise\n",
    "        should be V(s_T), the value function estimated for the last state.\n",
    "        This allows us to bootstrap the reward-to-go calculation to account\n",
    "        for timesteps beyond the arbitrary episode horizon (or epoch cutoff).\n",
    "        \"\"\"\n",
    "\n",
    "        path_slice = slice(self.path_start_idx, self.ptr)\n",
    "        rews = np.append(self.rew_buf[path_slice], last_val)\n",
    "        vals = np.append(self.val_buf[path_slice], last_val)\n",
    "\n",
    "        # the next two lines implement GAE-Lambda advantage calculation\n",
    "        deltas = rews[:-1] + self.gamma * vals[1:] - vals[:-1]\n",
    "        self.adv_buf[path_slice] = discount_cumsum(deltas, self.gamma * self.lam)\n",
    "\n",
    "        # the next line computes rewards-to-go, to be targets for the value function\n",
    "        self.ret_buf[path_slice] = discount_cumsum(rews, self.gamma)[:-1]\n",
    "\n",
    "        self.path_start_idx = self.ptr\n",
    "\n",
    "    def get(self): #取数据用于训练\n",
    "        \"\"\"\n",
    "        Call this at the end of an epoch to get all of the data from\n",
    "        the buffer, with advantages appropriately normalized (shifted to have\n",
    "        mean zero and std one). Also, resets some pointers in the buffer.\n",
    "        \"\"\"\n",
    "        assert self.ptr == self.max_size    # buffer has to be full before you can get\n",
    "        self.ptr, self.path_start_idx = 0, 0\n",
    "        # the next two lines implement the advantage normalization trick\n",
    "        adv_mean, adv_std = mpi_statistics_scalar(self.adv_buf)\n",
    "        self.adv_buf = (self.adv_buf - adv_mean) / adv_std\n",
    "        data = dict(obs=self.obs_buf, act=self.act_buf, ret=self.ret_buf,\n",
    "                    adv=self.adv_buf, logp=self.logp_buf)\n",
    "        \n",
    "        return {k: torch.as_tensor(v, dtype=torch.float32).to(device) for k,v in data.items()}\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### R4.2.3.2 定义ppo算法 及 更新策略"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义policy模型\n",
    "ac_pi = actor(4, act_dim, activation=nn.Tanh)  # 输入为观察值的channel（4）输出为action的深度（7）激活函数使用Tanh\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义value模型\n",
    "ac_v = critic(4, activation=nn.Tanh)  # 输入为观察值的channel（4）激活函数使用Tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up function for computing PPO policy loss\n",
    "def compute_loss_pi(data):\n",
    "    obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']\n",
    "\n",
    "    # Policy loss\n",
    "    pi, logp = ac_pi(obs, act) #计算action的概率分布\n",
    "    ratio = torch.exp(logp - logp_old) #计算新老策略的差异大小\n",
    "    clip_adv = torch.clamp(ratio, 1-clip_ratio, 1+clip_ratio) * adv #clip 新老策略的更新范围在（1-clip_ratio, 1+clip_ratio）内\n",
    "    loss_pi = -(torch.min(ratio * adv, clip_adv)).mean() #计算最终的policy loss 使adv（优势）更明显的方向做梯度更新\n",
    "\n",
    "    return loss_pi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up function for computing value loss\n",
    "def compute_loss_v(data):\n",
    "    obs, ret = data['obs'], data['ret']\n",
    "    return ((ac_v(obs) - ret)**2).mean() #真实值和预测值做均方差loss 使v模型预测更接近真实值\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up optimizers for policy and value function\n",
    "pi_optimizer = Adam(ac_pi.parameters(), lr=pi_lr)\n",
    "vf_optimizer = Adam(ac_v.parameters(), lr=vf_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#update 模型参数（训练）\n",
    "def update():\n",
    "    data = buf.get() #读取训练数据\n",
    "\n",
    "    # Train policy with multiple steps of gradient descent\n",
    "    for i in range(train_pi_iters):\n",
    "        pi_optimizer.zero_grad()\n",
    "        loss_pi, pi_info = compute_loss_pi(data)\n",
    "        loss_pi.backward()\n",
    "        pi_optimizer.step()\n",
    "\n",
    "    # Value function learning\n",
    "    for i in range(train_v_iters):\n",
    "        vf_optimizer.zero_grad()\n",
    "        loss_v = compute_loss_v(data)\n",
    "        loss_v.backward()\n",
    "        vf_optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare for interaction with environment\n",
    "o, ep_ret, ep_len = env.reset(), 0, 0\n",
    "# Main loop: collect experience in env and update/log each epoch\n",
    "for epoch in range(1000):\n",
    "    for t in range(2000):\n",
    "        with torch.no_grad(): #收集数据过程，不做参数更新\n",
    "            rr = torch.from_numpy(o.copy()).float().to(device) #数据转换\n",
    "            pi, _ = ac_pi(rr, None) # 计算pi\n",
    "            a = pi.sample() #取出action用于环境交互\n",
    "            logp = pi.log_prob(a) #去除a对应的概率存起来用于模型训练时判断更新幅度，防止步幅过大学习率过大\n",
    "            v = ac_v(torch.as_tensor(o, dtype=torch.float32).to(device)) # 计算v存起来给pi模型提供指引\n",
    "\n",
    "        next_o, r, d, _ = env.step(a.cpu().numpy().item()) #在游戏中执行模型输出的action\n",
    "        ep_ret += r\n",
    "        ep_len += 1\n",
    "\n",
    "        # save and log\n",
    "        buf.store(o, a.cpu().numpy(), r, v.cpu().numpy(), logp.cpu().numpy())\n",
    "\n",
    "        # Update obs (critical!)\n",
    "        o = next_o\n",
    "\n",
    "        #下面部分为对回合结束时的特殊情况做一下处理，比如最大步数达到了但是并没有gameover则需要获取下一帧的观察值，而小人game over时则不需要，初学者可以掠过\n",
    "        timeout = ep_len == max_ep_len\n",
    "        terminal = d \n",
    "        epoch_ended = t==local_steps_per_epoch-1\n",
    "\n",
    "        if terminal or epoch_ended:\n",
    "            if epoch_ended and not(terminal):\n",
    "                print('Warning: trajectory cut off by epoch at %d steps.'%ep_len, flush=True)\n",
    "            # if trajectory didn't reach terminal state, bootstrap value target\n",
    "            if epoch_ended:\n",
    "                print('epoch_end')\n",
    "                with torch.no_grad():\n",
    "                    v =ac_v(torch.from_numpy(o).float().to(device)).cpu().numpy()\n",
    "            else:\n",
    "                print('epret :',ep_ret)\n",
    "                v = 0\n",
    "            buf.finish_path(v)\n",
    "            if terminal:\n",
    "                # only save EpRet / EpLen if trajectory finished\n",
    "                logger.store(EpRet=ep_ret, EpLen=ep_len)\n",
    "            o, ep_ret, ep_len = env.reset(), 0, 0\n",
    "\n",
    "    # Perform PPO update!\n",
    "    update()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### R4.2.3.3 PPO算法完整代码（添加log记录、mpi多进程）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ppo函数完整代码\n",
    "def ppo(env_fn, actor=nn.Module, critic=nn.Module, ac_kwargs=dict(), seed=0,\n",
    "        steps_per_epoch=4000, epochs=50, gamma=0.99, clip_ratio=0.2, pi_lr=3e-4,\n",
    "        vf_lr=1e-3, train_pi_iters=80, train_v_iters=80, lam=0.97, max_ep_len=1000,\n",
    "        target_kl=0.01, logger_kwargs=dict(), save_freq=10):\n",
    "    # Special function to avoid certain slowdowns from PyTorch + MPI combo.\n",
    "    setup_pytorch_for_mpi()\n",
    "\n",
    "    # Set up logger and save configuration\n",
    "    logger = EpochLogger(**logger_kwargs)\n",
    "    logger.save_config(locals())\n",
    "\n",
    "    # Random seed\n",
    "    seed += 10000 * proc_id()\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    # Instantiate environment\n",
    "    env = env_fn()\n",
    "    obs_dim = env.observation_space.shape\n",
    "    act_dim = env.action_space.n\n",
    "    \n",
    "    # Create actor-critic module\n",
    "    ac_pi = actor(obs_dim[0], act_dim, hidden_sizes=[64, 64], activation=nn.Tanh)  # env.observation_space, env.action_space, nn.ReLU)\n",
    "    ac_v = critic(obs_dim[0], hidden_sizes=[64, 64], activation=nn.Tanh)  # env.observation_space, nn.ReLU)\n",
    "\n",
    "    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cup')\n",
    "    ac_pi.to(device)\n",
    "    ac_v.to(device)\n",
    "\n",
    "    # Sync params across processes\n",
    "    sync_params(ac_pi)\n",
    "    sync_params(ac_v)\n",
    "\n",
    "    # Count variables\n",
    "    def count_vars(module):\n",
    "        return sum([np.prod(p.shape) for p in module.parameters()])\n",
    "    var_counts = tuple(count_vars(module) for module in [ac_pi, ac_v])\n",
    "    logger.log('\\nNumber of parameters: \\t pi: %d, \\t v: %d\\n'%var_counts)\n",
    "\n",
    "    # Set up experience buffer\n",
    "    local_steps_per_epoch = int(steps_per_epoch / num_procs())\n",
    "    buf = PPOBuffer(obs_dim, env.action_space.shape, local_steps_per_epoch, gamma, lam)\n",
    "    \n",
    "    # Set up function for computing PPO policy loss\n",
    "    def compute_loss_pi(data):\n",
    "        obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']\n",
    "\n",
    "        # Policy loss\n",
    "        pi, logp = ac_pi(obs, act)\n",
    "        ratio = torch.exp(logp - logp_old)\n",
    "        clip_adv = torch.clamp(ratio, 1-clip_ratio, 1+clip_ratio) * adv\n",
    "        loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()\n",
    "\n",
    "        # Useful extra info\n",
    "        approx_kl = (logp_old - logp).mean().item()\n",
    "        ent = pi.entropy().mean().item()\n",
    "        clipped = ratio.gt(1+clip_ratio) | ratio.lt(1-clip_ratio)\n",
    "        clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()\n",
    "        pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)\n",
    "\n",
    "        return loss_pi, pi_info\n",
    "\n",
    "    # Set up function for computing value loss\n",
    "    def compute_loss_v(data):\n",
    "        obs, ret = data['obs'], data['ret']\n",
    "        return ((ac_v(obs) - ret)**2).mean()\n",
    "\n",
    "    # Set up optimizers for policy and value function\n",
    "    pi_optimizer = Adam(ac_pi.parameters(), lr=pi_lr)\n",
    "    vf_optimizer = Adam(ac_v.parameters(), lr=vf_lr)\n",
    "\n",
    "    # Set up model saving\n",
    "    logger.setup_pytorch_saver(ac_pi)\n",
    "    \n",
    "    def update():\n",
    "        data = buf.get()\n",
    "\n",
    "        pi_l_old, pi_info_old = compute_loss_pi(data)\n",
    "        pi_l_old = pi_l_old.item()\n",
    "        v_l_old = compute_loss_v(data).item()\n",
    "\n",
    "        # Train policy with multiple steps of gradient descent\n",
    "        for i in range(train_pi_iters):\n",
    "            pi_optimizer.zero_grad()\n",
    "            loss_pi, pi_info = compute_loss_pi(data)\n",
    "            kl = mpi_avg(pi_info['kl'])\n",
    "            if kl > 1.5 * target_kl:\n",
    "                logger.log('Early stopping at step %d due to reaching max kl.'%i)\n",
    "                break\n",
    "            loss_pi.backward()\n",
    "            mpi_avg_grads(ac_pi)    # average grads across MPI processes\n",
    "            pi_optimizer.step()\n",
    "\n",
    "        logger.store(StopIter=i)\n",
    "\n",
    "        # Value function learning\n",
    "        for i in range(train_v_iters):\n",
    "            vf_optimizer.zero_grad()\n",
    "            loss_v = compute_loss_v(data)\n",
    "            loss_v.backward()\n",
    "            mpi_avg_grads(ac_v)    # average grads across MPI processes\n",
    "            vf_optimizer.step()\n",
    "\n",
    "        # Log changes from update\n",
    "        kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']\n",
    "        logger.store(LossPi=pi_l_old, LossV=v_l_old,\n",
    "                     KL=kl, Entropy=ent, ClipFrac=cf,\n",
    "                     DeltaLossPi=(loss_pi.item() - pi_l_old),\n",
    "                     DeltaLossV=(loss_v.item() - v_l_old))\n",
    "\n",
    "    # Prepare for interaction with environment\n",
    "    start_time = time.time()\n",
    "    o, ep_ret, ep_len = env.reset(), 0, 0\n",
    "\n",
    "    # Main loop: collect experience in env and update/log each epoch\n",
    "    for epoch in range(epochs):\n",
    "        for t in range(local_steps_per_epoch):\n",
    "            # a, v, logp = ac.step(torch.as_tensor(o, dtype=torch.float32))\n",
    "            with torch.no_grad():\n",
    "                rr = torch.from_numpy(o.copy()).float().to(device)#.unsqueeze(0)\n",
    "                pi, _ = ac_pi(rr, None)\n",
    "                a = pi.sample()\n",
    "                # logp_a = self.pi._log_prob_from_distribution(pi, a)\n",
    "                logp = pi.log_prob(a)#.sum(axis=-1)\n",
    "                v = ac_v(torch.as_tensor(o, dtype=torch.float32).to(device))\n",
    "\n",
    "            next_o, r, d, _ = env.step(a.cpu().numpy().item())\n",
    "            ep_ret += r\n",
    "            ep_len += 1\n",
    "\n",
    "            # save and log\n",
    "            buf.store(o, a.cpu().numpy(), r, v.cpu().numpy(), logp.cpu().numpy())\n",
    "            logger.store(VVals=v.cpu().numpy())\n",
    "\n",
    "            # Update obs (critical!)\n",
    "            o = next_o\n",
    "\n",
    "            timeout = ep_len == max_ep_len\n",
    "            terminal = d #or timeout\n",
    "            epoch_ended = t==local_steps_per_epoch-1\n",
    "\n",
    "            if terminal or epoch_ended:\n",
    "                if epoch_ended and not(terminal):\n",
    "                    print('Warning: trajectory cut off by epoch at %d steps.'%ep_len, flush=True)\n",
    "                # if trajectory didn't reach terminal state, bootstrap value target\n",
    "                if epoch_ended:\n",
    "                    print('epoch_end')\n",
    "                    # _, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))\n",
    "                    with torch.no_grad():\n",
    "                        v =ac_v(torch.from_numpy(o).float().to(device)).cpu().numpy()\n",
    "                else:\n",
    "                    print('epret :',ep_ret)\n",
    "                    v = 0\n",
    "                buf.finish_path(v)\n",
    "                if terminal:\n",
    "                    # only save EpRet / EpLen if trajectory finished\n",
    "                    logger.store(EpRet=ep_ret, EpLen=ep_len)\n",
    "                o, ep_ret, ep_len = env.reset(), 0, 0\n",
    "\n",
    "\n",
    "        # Save model\n",
    "        if (epoch % save_freq == 0) or (epoch == epochs-1):\n",
    "            logger.save_state({'env': env}, None)\n",
    "\n",
    "        # Perform PPO update!\n",
    "        update()\n",
    "\n",
    "        # Log info about epoch\n",
    "        logger.log_tabular('Epoch', epoch)\n",
    "        logger.log_tabular('EpRet', with_min_and_max=True)\n",
    "        logger.log_tabular('EpLen', average_only=True)\n",
    "        logger.log_tabular('VVals', with_min_and_max=True)\n",
    "        logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch)\n",
    "        logger.log_tabular('LossPi', average_only=True)\n",
    "        logger.log_tabular('LossV', average_only=True)\n",
    "        logger.log_tabular('DeltaLossPi', average_only=True)\n",
    "        logger.log_tabular('DeltaLossV', average_only=True)\n",
    "        logger.log_tabular('Entropy', average_only=True)\n",
    "        logger.log_tabular('KL', average_only=True)\n",
    "        logger.log_tabular('ClipFrac', average_only=True)\n",
    "        logger.log_tabular('StopIter', average_only=True)\n",
    "        logger.log_tabular('Time', time.time()-start_time)\n",
    "        logger.dump_tabular()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### R4.2.3.3 主函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: Log dir /root/lele/spinningup/spinningup/data/ppo/ppo_s0 already exists! Storing info there anyway.\n",
      "\u001b[32;1mLogging data to /root/lele/spinningup/spinningup/data/ppo/ppo_s0/progress.txt\u001b[0m\n",
      "dict_items([('env_fn', <function <lambda> at 0x7fc32ecbc7a0>), ('actor', <class '__main__.userActor'>), ('critic', <class '__main__.userCritic'>), ('ac_kwargs', {'hidden_sizes': 128}), ('seed', 0), ('steps_per_epoch', 10000), ('epochs', 150), ('gamma', 0.999), ('pi_lr', 0.001), ('vf_lr', 0.001), ('lam', 0.97), ('max_ep_len', 1000), ('logger_kwargs', {'output_dir': '/root/lele/spinningup/spinningup/data/ppo/ppo_s0', 'exp_name': 'ppo'}), ('save_freq', 10), ('clip_ratio', 0.2), ('logger', <spinup.utils.logx.EpochLogger object at 0x7fc32fe81510>), ('target_kl', 0.01), ('train_pi_iters', 80), ('train_v_iters', 80)])\n",
      "\u001b[36;1mSaving config:\n",
      "\u001b[0m\n",
      "{\n",
      "    \"ac_kwargs\":\t{\n",
      "        \"hidden_sizes\":\t128\n",
      "    },\n",
      "    \"actor\":\t\"userActor\",\n",
      "    \"clip_ratio\":\t0.2,\n",
      "    \"critic\":\t\"userCritic\",\n",
      "    \"env_fn\":\t\"<function <lambda> at 0x7fc32ecbc7a0>\",\n",
      "    \"epochs\":\t150,\n",
      "    \"exp_name\":\t\"ppo\",\n",
      "    \"gamma\":\t0.999,\n",
      "    \"lam\":\t0.97,\n",
      "    \"logger\":\t{\n",
      "        \"<spinup.utils.logx.EpochLogger object at 0x7fc32fe81510>\":\t{\n",
      "            \"epoch_dict\":\t{},\n",
      "            \"exp_name\":\t\"ppo\",\n",
      "            \"first_row\":\ttrue,\n",
      "            \"log_current_row\":\t{},\n",
      "            \"log_headers\":\t[],\n",
      "            \"output_dir\":\t\"/root/lele/spinningup/spinningup/data/ppo/ppo_s0\",\n",
      "            \"output_file\":\t{\n",
      "                \"<_io.TextIOWrapper name='/root/lele/spinningup/spinningup/data/ppo/ppo_s0/progress.txt' mode='w' encoding='UTF-8'>\":\t{\n",
      "                    \"mode\":\t\"w\"\n",
      "                }\n",
      "            }\n",
      "        }\n",
      "    },\n",
      "    \"logger_kwargs\":\t{\n",
      "        \"exp_name\":\t\"ppo\",\n",
      "        \"output_dir\":\t\"/root/lele/spinningup/spinningup/data/ppo/ppo_s0\"\n",
      "    },\n",
      "    \"max_ep_len\":\t1000,\n",
      "    \"pi_lr\":\t0.001,\n",
      "    \"save_freq\":\t10,\n",
      "    \"seed\":\t0,\n",
      "    \"steps_per_epoch\":\t10000,\n",
      "    \"target_kl\":\t0.01,\n",
      "    \"train_pi_iters\":\t80,\n",
      "    \"train_v_iters\":\t80,\n",
      "    \"vf_lr\":\t0.001\n",
      "}\n",
      "cnn_model(\n",
      "  (conv1): Conv2d(4, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv3): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv4): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (liner): Linear(in_features=1152, out_features=512, bias=True)\n",
      "  (fc_out): Linear(in_features=512, out_features=12, bias=True)\n",
      ")\n",
      "cnn_model(\n",
      "  (conv1): Conv2d(4, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv3): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (conv4): Conv2d(32, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "  (liner): Linear(in_features=1152, out_features=512, bias=True)\n",
      "  (fc_out): Linear(in_features=512, out_features=1, bias=True)\n",
      ")\n",
      "\u001b[32;1m\n",
      "Number of parameters: \t pi: 625420, \t v: 619777\n",
      "\u001b[0m\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'Categorical' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-dc98eabedcf5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mac_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_sizes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhid_sizes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgamma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgamma\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0mseed\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mseed\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m         logger_kwargs=logger_kwargs, clip_ratio=0.2, pi_lr=0.001, vf_lr=0.001)\n\u001b[0m\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-cbb102b3b19d>\u001b[0m in \u001b[0;36mppo\u001b[0;34m(env_fn, actor, critic, ac_kwargs, seed, steps_per_epoch, epochs, gamma, clip_ratio, pi_lr, vf_lr, train_pi_iters, train_v_iters, lam, max_ep_len, target_kl, logger_kwargs, save_freq)\u001b[0m\n\u001b[1;32m    119\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m                 \u001b[0mrr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#.unsqueeze(0)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m                 \u001b[0mpi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mac_pi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m                 \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m                 \u001b[0;31m# logp_a = self.pi._log_prob_from_distribution(pi, a)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    540\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 541\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    542\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    543\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-12-6cc7d8225054>\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, obs, act)\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m#计算策略分布，和action的概率\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mact\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0mpi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCategorical\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogits_net\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0mlogp_a\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mact\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Categorical' is not defined"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    \n",
    "    hid_sizes = 128\n",
    "    gamma = 0.999 #reward 的衰减因子\n",
    "    seed = 0\n",
    "    steps = 10000\n",
    "    epochs = 150\n",
    "    cpu = 1\n",
    "    exp_name = \"ppo\"\n",
    "    \n",
    "    \n",
    "    import os\n",
    "    os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\n",
    "    mpi_fork(cpu)  # run parallel code with mpi\n",
    "\n",
    "    from spinup.utils.run_utils import setup_logger_kwargs\n",
    "    logger_kwargs = setup_logger_kwargs(exp_name, seed)\n",
    "    env_fn = lambda : create_train_env(1,1,'complex')\n",
    "    ppo(env_fn, actor=userActor, critic=userCritic,\n",
    "        ac_kwargs=dict(hidden_sizes=hid_sizes), gamma=gamma,\n",
    "        seed=seed, steps_per_epoch=steps, epochs=epochs,\n",
    "        logger_kwargs=logger_kwargs, clip_ratio=0.2, pi_lr=0.001, vf_lr=0.001)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### R4.2.3.4 查看训练过程指标\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/root/lele/king_ppo\n",
      "Plotting from...\n",
      "==================================================\n",
      "\n",
      "/root/lele/spinningup/spinningup/data/ppo/ppo_s0\n",
      "\n",
      "==================================================\n",
      "Could not read from /root/lele/spinningup/spinningup/data/ppo/ppo_s0/progress.txt\n",
      "Traceback (most recent call last):\n",
      "  File \"/root/lele/spinningup/spinningup/spinup/utils/plot.py\", line 233, in <module>\n",
      "    main()\n",
      "  File \"/root/lele/spinningup/spinningup/spinup/utils/plot.py\", line 230, in main\n",
      "    estimator=args.est)\n",
      "  File \"/root/lele/spinningup/spinningup/spinup/utils/plot.py\", line 162, in make_plots\n",
      "    plot_data(data, xaxis=xaxis, value=value, condition=condition, smooth=smooth, estimator=estimator)\n",
      "  File \"/root/lele/spinningup/spinningup/spinup/utils/plot.py\", line 31, in plot_data\n",
      "    data = pd.concat(data, ignore_index=True)\n",
      "  File \"/root/anaconda3/lib/python3.7/site-packages/pandas/core/reshape/concat.py\", line 255, in concat\n",
      "    sort=sort,\n",
      "  File \"/root/anaconda3/lib/python3.7/site-packages/pandas/core/reshape/concat.py\", line 304, in __init__\n",
      "    raise ValueError(\"No objects to concatenate\")\n",
      "ValueError: No objects to concatenate\n",
      "Traceback (most recent call last):\n",
      "  File \"/root/anaconda3/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"/root/anaconda3/lib/python3.7/runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"/root/lele/spinningup/spinningup/spinup/run.py\", line 243, in <module>\n",
      "    subprocess.check_call(args, env=os.environ)\n",
      "  File \"/root/anaconda3/lib/python3.7/subprocess.py\", line 347, in check_call\n",
      "    raise CalledProcessError(retcode, cmd)\n",
      "subprocess.CalledProcessError: Command '['/root/anaconda3/bin/python', '/root/lele/spinningup/spinningup/spinup/utils/plot.py', '/root/lele/spinningup/spinningup/data/ppo/ppo_s0']' returned non-zero exit status 1.\n"
     ]
    }
   ],
   "source": [
    "#查看训练plot出来\n",
    "!pwd\n",
    "%matplotlib inline\n",
    "!python -m spinup.run plot /root/lele/spinningup/spinningup/data/ppo/ppo_s0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### R4.2.3.5 加载训练好的模型并在游戏中运行"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cup')\n",
    "def load_pytorch_policy(fpath, itr='', deterministic=False):\n",
    "    \"\"\" Load a pytorch policy saved with Spinning Up Logger.\"\"\"\n",
    "\n",
    "    fname = osp.join(fpath, 'pyt_save', 'model' + itr + '.pt')\n",
    "    print('\\n\\nLoading from %s.\\n\\n' % fname)\n",
    "\n",
    "    model = torch.load(fname) #加载训练好的模型\n",
    "\n",
    "    # make function for producing an action given a single state\n",
    "    def get_action(x):\n",
    "        with torch.no_grad():\n",
    "            x = torch.as_tensor(x, dtype=torch.float32)\n",
    "            pi, _ = model(x.to(device), None)\n",
    "            action = pi.sample()\n",
    "        return action.cpu()\n",
    "\n",
    "    return get_action\n",
    "\n",
    "\n",
    "def run_policy(env, get_action, max_ep_len=None, num_episodes=100, render=True):\n",
    "    assert env is not None, \\\n",
    "        \"Environment not found!\\n\\n It looks like the environment wasn't saved, \" + \\\n",
    "        \"and we can't run the agent in it. :( \\n\\n Check out the readthedocs \" + \\\n",
    "        \"page on Experiment Outputs for how to handle this situation.\"\n",
    "\n",
    "    logger = EpochLogger()\n",
    "\n",
    "    o, r, d, ep_ret, ep_len, n = env.reset(), 0, False, 0, 0, 0\n",
    "    while n < num_episodes:\n",
    "        if render:\n",
    "            env.render()\n",
    "            time.sleep(1e-3)\n",
    "\n",
    "        a = get_action(o)\n",
    "        o, r, d, _ = env.step(a.numpy().item())\n",
    "        ep_ret += r\n",
    "        ep_len += 1\n",
    "\n",
    "        if d or (ep_len == max_ep_len):\n",
    "            logger.store(EpRet=ep_ret, EpLen=ep_len)\n",
    "            print('Episode %d \\t EpRet %.3f \\t EpLen %d' % (n, ep_ret, ep_len))\n",
    "            o, r, d, ep_ret, ep_len = env.reset(), 0, False, 0, 0\n",
    "            n += 1\n",
    "\n",
    "    logger.log_tabular('EpRet', with_min_and_max=True)\n",
    "    logger.log_tabular('EpLen', average_only=True)\n",
    "    logger.dump_tabular()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    \n",
    "    #这里根据你自己的spriningup安装路径来修改\n",
    "    fpath = r'/root/lele/spinningup/spinningup/data/ppo/ppo_s0/'\n",
    "    episodes = 100\n",
    "    store_true = False\n",
    "    \n",
    "    env = create_train_env(1,1, 'complx')\n",
    "    get_action = load_pytorch_policy(fpath)#itr='_50'\n",
    "    run_policy(env, get_action, 0, episodes, store_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "注：完整代码见：https://github.com/gaoxiaos/Supermariobros-PPO-pytorch.git"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R5 强化学习的近况\n",
    "强化学习一直被学术届认为是通往通用智能的大门，所以在这个领域深耕的学术论文每年都在指数增加，特别是今年各大AI会议的论坛都把强化学习的讨论放在了重要位置，比如世界人工智能大会的主论坛、ijcai今年在清华平台举办的麻将AI大赛，nips更是把四个赛道全部放在了强化学习领域\n",
    "强化学习面临的问题还有很多，比如数据采样的难度，由于数据来源于交互所以模型的学习速度依赖于采样速度，如何在有限的交互步数下取得更好的成绩就成了业内模型创新的方向。再如很多场景无法明确提出奖励函数，这时候如何让模型模仿专家达到专业的程度等各方面的研究都在进行，以及算法方面，自从ppo出来并一统江湖后已经很长时间没有出现质的飞跃的算法，这块的研究也是非常值得进一步探索的。如果你也有兴趣，那么欢迎添加微信入群，和其他小伙伴一起交流探索，打比赛拿奖金。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R5 挑战一下"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过上面的学习，我们来挑战下有一定难度的新游戏“大鱼吃小鱼”-该游戏是openai新推出的一个“随机生成”环境procgen里非常有代表性的一个场景，为了解决网络模型通常的“记住”怎么走而非完全理解怎么走的问题，openai推出了procgen benchmark用来评估模型的泛化性能，每次reset游戏时游戏的分布都是随机生成的，比如大鱼吃小鱼，每次开场的小鱼分布都是随机的，小鱼行为也都是随机的，这样就需要你控制的angent要真实理解周围环境才能作出正确判断吃到更多小鱼。\n",
    "竞赛直达："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
